###############################################################
## check job stauts and retry failed jobs ##
## usage: ./retry_jobs.sh $1
##        arg1: history file (contains jobID, output, Njobs) 
## Lailin, Ann Arbor, 1.2013
###############################################################

Retry_dir=Retry
if [ ! -d $Retry_dir ]; then mkdir $Retry_dir; fi

timetag=`date +%Y%m%d%H%M`
check_jobID=-1
nFinished=-1
checkFailedID() {
    pbook -c "show($1)"  > tmp_status
    less tmp_status >> out_$timetag
    jobStatus=tmp_status
    nFailed=`less $jobStatus | grep failed | awk '{print $3}'`
    nCancelled=`less $jobStatus | grep cancelled | awk '{print $3}'`
    nFinished=`less $jobStatus | grep finished | awk '{print $3}'`
    if [ -z "${nFinished}" ]; then nFinished=0; fi
    echo "failed: $nFailed; cancelled: $nCancelled; finished: $nFinished"
    if [ -n "${nFailed}" ] || [ -n "${nCancelled}" ]; then
        retrySetID=`less $jobStatus | grep retrySetID | awk '{print $3}'`
        retryID=`less $jobStatus | grep retryID | awk '{print $3}'`
        if [ -n "${retryID}" ] && [ "${retryID}" != "0" ]; then
            checkFailedID $retryID
        elif [ -n "${retrySetID}" ] && [ "${retrySetID}" != "0" ]; then
            checkFailedID $retrySetID
        else 
            check_jobID=$1
        fi
    else 
        check_jobID=-1    
    fi

}

pbook -c "sync()"
#pbook -c "show(-500)" > $Retry_dir/out_frozen_$timetag
#less $Retry_dir/out_frozen_$timetag | grep JobsetID | awk '{ print $3 }'  > $Retry_dir/frozen_jobID_$timetag

#for job in `cat $Retry_dir/frozen_jobID_$timetag`; do
while read job output Njobs; do
    if [ -z "${job}" ] || [ -z "${output}" ] || [ -z "${Njobs}" ]; then
        continue
    fi
    if [ "Njobs" == "FAIL" ]; then
        echo "Warning => Failed jobs: $output"
        continue
    fi
    check_jobID=$job
    nFinished=-1
    checkFailedID $job
    echo "failed_jobID: $check_jobID"
    if [ "${check_jobID}" == "-1" ]; then
        echo "Succeeded: $job" >> $Retry_dir/good_job_$timetag
    else 
        echo "Failed: $check_jobID" >> $Retry_dir/failed_job_$timetag
        if [ ! "${nFinished}" == "0" ]; then 
            #pbook -c "retry($check_jobID, newSite=1)" >> out_retry_$timetag
            pbook -c "retry($check_jobID)" >> $Retry_dir/out_retry_$timetag
            #pbook -c "retry($check_jobID, retryBuild=1)" >> $Retry_dir/out_retry_$timetag
        else
            #pbook -c "retry($check_jobID, newSite=1, retryBuild=1)" >> out_retry_$timetag
            pbook -c "retry($check_jobID, retryBuild=1)" >> $Retry_dir/out_retry_$timetag
        fi
        if [ $? -eq 0 ]; then
            echo "Resubmitted: $check_jobID" >> $Retry_dir/retried_job_$timetag
        else
            echo "Resubmitted but failed: $check_jobID" >> $Retry_dir/failed_retried_job_$timetag
        fi
    fi
done < Retry/failed_job_201301072342_retry
